{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e7b1ed33-755c-4af7-b384-6e4a7079ac94",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install SwissArmyTransformer>=0.3.6 torch>1.10.0 torchvision transformers>=4.27.1 cpm_kernels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6b15308c-2abf-452a-a2c2-a87ee56890da",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: accelerate==0.20.3 in /opt/conda/lib/python3.10/site-packages (0.20.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.20.3) (1.23.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.20.3) (23.1)\n",
      "Requirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate==0.20.3) (5.9.5)\n",
      "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate==0.20.3) (5.4.1)\n",
      "Requirement already satisfied: torch>=1.6.0 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.20.3) (2.0.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->accelerate==0.20.3) (3.12.0)\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->accelerate==0.20.3) (4.5.0)\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->accelerate==0.20.3) (1.11.1)\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->accelerate==0.20.3) (3.1)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->accelerate==0.20.3) (3.1.2)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.6.0->accelerate==0.20.3) (2.1.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.6.0->accelerate==0.20.3) (1.3.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install accelerate==0.20.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8196bc85-816e-4b23-bd30-67b3c5197550",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pip in /opt/conda/lib/python3.10/site-packages (23.1.2)\n",
      "Collecting pip\n",
      "  Using cached pip-23.2.1-py3-none-any.whl (2.1 MB)\n",
      "Installing collected packages: pip\n",
      "  Attempting uninstall: pip\n",
      "    Found existing installation: pip 23.1.2\n",
      "    Uninstalling pip-23.1.2:\n",
      "      Successfully uninstalled pip-23.1.2\n",
      "Successfully installed pip-23.2.1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install --upgrade pip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2636aa6f-ca1c-4e19-8fb8-d608667e76e8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "[2023-08-11 06:59:49,817] [INFO] [RANK 0] > initializing model parallel with size 1\n",
      "[2023-08-11 06:59:49,819] [INFO] [RANK 0] You are using model-only mode.\n",
      "For torch.distributed users or loading model parallel models, set environment variables RANK, WORLD_SIZE and LOCAL_RANK.\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/init.py:405: UserWarning: Initializing zero-element tensors is a no-op\n",
      "  warnings.warn(\"Initializing zero-element tensors is a no-op\")\n",
      "Loading checkpoint shards: 100%|██████████| 5/5 [01:05<00:00, 13.04s/it]\n"
     ]
    }
   ],
   "source": [
    "# from transformers import AutoTokenizer, AutoModel\n",
    "# tokenizer = AutoTokenizer.from_pretrained(\"THUDM/visualglm-6b\", trust_remote_code=True)\n",
    "# model = AutoModel.from_pretrained(\"THUDM/visualglm-6b\", trust_remote_code=True).quantize(8).half().cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "54c8acd7-57b6-4196-b673-749a8e0ac6e2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Specify both input_ids and inputs_embeds at the same time, will use inputs_embeds\n"
     ]
    }
   ],
   "source": [
    "# image_path = \"http://images.cocodataset.org/val2017/000000039769.jpg\"\n",
    "# prompt = \"里面有几只猫？\"\n",
    "# # prompt = \"Question: please describe this image. Answer:\"\n",
    "# response,_ = model.chat(tokenizer, image_path,prompt , history=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6756846a-3553-430a-a059-9bd77d8055ce",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "有两只猫。它们睡在粉色沙发上，靠近电视遥控器。它们看起来很放松和舒适。图片中的猫咪似乎很享受彼此的陪伴。它们可能正在休息或睡觉，或者只是在那里玩耍。这为沙发增添了温馨的氛围。此外，两个猫的毛皮被覆盖在粉色的沙发上，表明它们在这个地方感到温暖和安全。\n"
     ]
    }
   ],
   "source": [
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c0b705e5-bceb-44cc-89b2-28917b696d2d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# text = f\"<img>{image_path}</img>\"\n",
    "# # model.process_image(text)\n",
    "# inputs  = model.build_inputs_with_image(tokenizer,image_path, prompt,history=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9281597f-ebf2-4c87-95d5-924026d2e3cc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[    46,   2265,     33,      0,      0,      0,      0,      0,      0,\n",
       "               0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
       "               0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
       "               0,      0,      0,      0,      0,      0,      0,      0,     98,\n",
       "            2265,     33,  64286,     12,   1095,     24,   8186,      4,  67342,\n",
       "              12, 130001, 130004]], device='cuda:0'),\n",
       " 'pre_image_length': 3,\n",
       " 'images': tensor([[[[ 0.2808,  0.3828,  0.4268,  ..., -0.2886, -0.2739, -0.2886],\n",
       "           [ 0.3245,  0.3828,  0.4121,  ..., -0.2886, -0.2886, -0.3179],\n",
       "           [ 0.2808,  0.3538,  0.3684,  ..., -0.3762, -0.3469, -0.3179],\n",
       "           ...,\n",
       "           [ 1.6387,  1.5361,  1.4189,  ...,  1.3906,  1.2881,  1.2441],\n",
       "           [ 1.6094,  1.5508,  1.5068,  ...,  1.2148,  0.9814,  0.8501],\n",
       "           [ 1.6094,  1.4775,  1.4922,  ...,  0.1201, -0.1281, -0.3909]],\n",
       " \n",
       "          [[-1.3916, -1.3916, -1.3916,  ..., -1.5420, -1.5420, -1.5566],\n",
       "           [-1.3467, -1.3467, -1.3467,  ..., -1.5273, -1.5117, -1.5273],\n",
       "           [-1.4072, -1.3770, -1.3467,  ..., -1.5566, -1.5420, -1.5420],\n",
       "           ...,\n",
       "           [-0.3413, -0.4614, -0.5513,  ..., -0.6416, -0.7017, -0.7466],\n",
       "           [-0.3413, -0.3865, -0.4915,  ..., -0.7314, -0.8667, -0.9268],\n",
       "           [-0.3713, -0.4915, -0.5063,  ..., -1.2871, -1.3770, -1.4824]],\n",
       " \n",
       "          [[-0.6553, -0.4990, -0.5415,  ..., -1.0107, -0.9258, -0.9541],\n",
       "           [-0.6982, -0.5986, -0.5703,  ..., -1.0107, -0.9541, -1.0107],\n",
       "           [-0.6128, -0.5273, -0.4990,  ..., -1.0254, -1.0391, -1.0537],\n",
       "           ...,\n",
       "           [ 1.3643,  1.3496,  1.1221,  ...,  1.1650,  1.0654,  0.9512],\n",
       "           [ 1.3350,  1.1787,  1.2646,  ...,  0.9370,  0.7524,  0.6245],\n",
       "           [ 1.3779,  1.3779,  1.2646,  ..., -0.1293, -0.5562, -0.7407]]]],\n",
       "        device='cuda:0', dtype=torch.float16)}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f8c5d21-3629-408d-bed1-34354cd760c9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "df28cd30-c41c-44ed-a0d2-49000b7ffa7e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "有两只猫。它们睡在粉色沙发上，靠近电视遥控器。图片中有一只棕色的老虎和一只橙色的猫。这两只猫看起来很放松，似乎正在享受彼此的陪伴。他们可能正在睡觉或休息，或者只是简单地躺在一起。这只猫在沙发旁边，另一只猫则在床上，这表明它们可能是同一家庭的宠物。\n"
     ]
    }
   ],
   "source": [
    "# print(response)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5660b06-00e1-4436-8293-b3b3381c626e",
   "metadata": {},
   "source": [
    "## 部署到Sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1bbbd74b-132c-4c2c-83c0-0f78a0213c74",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install huggingface-hub -Uq\n",
    "!pip install -Uq sagemaker "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d9ab7c07-0944-4c3c-929f-81f5e5b2fe54",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n",
      "Processing ./code/botocore-1.29.157-py3-none-any.whl\n",
      "Processing ./code/boto3-1.26.157-py3-none-any.whl\n",
      "Collecting jmespath<2.0.0,>=0.7.1 (from botocore==1.29.157)\n",
      "  Using cached jmespath-1.0.1-py3-none-any.whl (20 kB)\n",
      "Collecting python-dateutil<3.0.0,>=2.1 (from botocore==1.29.157)\n",
      "  Using cached python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)\n",
      "Collecting urllib3<1.27,>=1.25.4 (from botocore==1.29.157)\n",
      "  Obtaining dependency information for urllib3<1.27,>=1.25.4 from https://files.pythonhosted.org/packages/c5/05/c214b32d21c0b465506f95c4f28ccbcba15022e000b043b72b3df7728471/urllib3-1.26.16-py2.py3-none-any.whl.metadata\n",
      "  Downloading urllib3-1.26.16-py2.py3-none-any.whl.metadata (48 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.4/48.4 kB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting s3transfer<0.7.0,>=0.6.0 (from boto3==1.26.157)\n",
      "  Using cached s3transfer-0.6.1-py3-none-any.whl (79 kB)\n",
      "Collecting six>=1.5 (from python-dateutil<3.0.0,>=2.1->botocore==1.29.157)\n",
      "  Using cached six-1.16.0-py2.py3-none-any.whl (11 kB)\n",
      "Using cached urllib3-1.26.16-py2.py3-none-any.whl (143 kB)\n",
      "Installing collected packages: urllib3, six, jmespath, python-dateutil, botocore, s3transfer, boto3\n",
      "  Attempting uninstall: urllib3\n",
      "    Found existing installation: urllib3 1.26.15\n",
      "    Uninstalling urllib3-1.26.15:\n",
      "      Successfully uninstalled urllib3-1.26.15\n",
      "  Attempting uninstall: six\n",
      "    Found existing installation: six 1.16.0\n",
      "    Uninstalling six-1.16.0:\n",
      "      Successfully uninstalled six-1.16.0\n",
      "  Attempting uninstall: jmespath\n",
      "    Found existing installation: jmespath 1.0.1\n",
      "    Uninstalling jmespath-1.0.1:\n",
      "      Successfully uninstalled jmespath-1.0.1\n",
      "  Attempting uninstall: python-dateutil\n",
      "    Found existing installation: python-dateutil 2.8.2\n",
      "    Uninstalling python-dateutil-2.8.2:\n",
      "      Successfully uninstalled python-dateutil-2.8.2\n",
      "  Attempting uninstall: botocore\n",
      "    Found existing installation: botocore 1.29.132\n",
      "    Uninstalling botocore-1.29.132:\n",
      "      Successfully uninstalled botocore-1.29.132\n",
      "  Attempting uninstall: s3transfer\n",
      "    Found existing installation: s3transfer 0.6.1\n",
      "    Uninstalling s3transfer-0.6.1:\n",
      "      Successfully uninstalled s3transfer-0.6.1\n",
      "  Attempting uninstall: boto3\n",
      "    Found existing installation: boto3 1.26.132\n",
      "    Uninstalling boto3-1.26.132:\n",
      "      Successfully uninstalled boto3-1.26.132\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "awscli 1.27.132 requires botocore==1.29.132, but you have botocore 1.29.157 which is incompatible.\n",
      "awscli 1.27.132 requires PyYAML<5.5,>=3.10, but you have pyyaml 6.0.1 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed boto3-1.26.157 botocore-1.29.157 jmespath-1.0.1 python-dateutil-2.8.2 s3transfer-0.6.1 six-1.16.0 urllib3-1.26.16\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install sagemaker pip --upgrade  --quiet\n",
    "# Note the following may error depending on which awscli is installed in your jupyter kernel, \n",
    "# but that is ok \n",
    "%pip install ./code/botocore-1.29.157-py3-none-any.whl ./code/boto3-1.26.157-py3-none-any.whl --force"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4147ccf0-4759-43f2-bc64-6a5caf4b45c5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "from pathlib import Path\n",
    "\n",
    "local_model_path = Path(\"./LLM_visualglm_stream_model\")\n",
    "local_model_path.mkdir(exist_ok=True)\n",
    "model_name = \"THUDM/visualglm-6b\"\n",
    "commit_hash = \"f4f759acde0926fefcd35e2c626e08adb452eff8\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aaac9225-7cf3-4a10-8543-ad1528b9305c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "snapshot_download(repo_id=model_name, cache_dir=local_model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71974a2d-2443-4e2b-b310-8ba754db92fd",
   "metadata": {},
   "source": [
    "### 2. 把模型拷贝到S3为后续部署做准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "279dad99-a6bc-4b53-9d13-9b49f17103d7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "from sagemaker import image_uris\n",
    "import boto3\n",
    "import os\n",
    "import time\n",
    "import json\n",
    "\n",
    "role = sagemaker.get_execution_role()  # execution role for the endpoint\n",
    "sess = sagemaker.session.Session()  # sagemaker session for interacting with different AWS APIs\n",
    "bucket = sess.default_bucket()  # bucket to house artifacts\n",
    "\n",
    "region = sess._region_name\n",
    "account_id = sess.account_id()\n",
    "\n",
    "s3_client = boto3.client(\"s3\")\n",
    "sm_client = boto3.client(\"sagemaker\")\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e11f7b37-fc8f-44b0-a8ec-a49850ae3a7e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3_code_prefix: LLM-RAG/workshop/LLM_visualglm_deploy_code\n",
      "model_snapshot_path: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8\n"
     ]
    }
   ],
   "source": [
    "s3_model_prefix = \"LLM-RAG/workshop/LLM_visualglm_stream_model\"  # folder where model checkpoint will go\n",
    "model_snapshot_path = list(local_model_path.glob(\"**/snapshots/*\"))[0]\n",
    "s3_code_prefix = \"LLM-RAG/workshop/LLM_visualglm_deploy_code\"\n",
    "print(f\"s3_code_prefix: {s3_code_prefix}\")\n",
    "print(f\"model_snapshot_path: {model_snapshot_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "560c44ed-6ca4-4caa-b596-92f95da01849",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/MODEL_LICENSE to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/MODEL_LICENSE\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/README.md to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/README.md\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/config.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/config.json\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/.gitattributes to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/.gitattributes\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/configuration_chatglm.py to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/configuration_chatglm.py\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/LICENSE to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/LICENSE\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/ice_text.model to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/ice_text.model\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/modeling_chatglm.py to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/modeling_chatglm.py\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/pytorch_model-00005-of-00005.bin to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/pytorch_model-00005-of-00005.bin\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/pytorch_model.bin.index.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/pytorch_model.bin.index.json\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/quantization.py to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/quantization.py\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/tokenization_chatglm.py to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/tokenization_chatglm.py\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/tokenizer_config.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/tokenizer_config.json\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/visual.py to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/visual.py\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/pytorch_model-00003-of-00005.bin to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/pytorch_model-00003-of-00005.bin\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/pytorch_model-00002-of-00005.bin to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/pytorch_model-00002-of-00005.bin\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/pytorch_model-00004-of-00005.bin to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/pytorch_model-00004-of-00005.bin\n",
      "upload: LLM_visualglm_stream_model/models--THUDM--visualglm-6b/snapshots/f4f759acde0926fefcd35e2c626e08adb452eff8/pytorch_model-00001-of-00005.bin to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/pytorch_model-00001-of-00005.bin\n"
     ]
    }
   ],
   "source": [
    "!aws s3 cp --recursive {model_snapshot_path} s3://{bucket}/{s3_model_prefix}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eca3d7fc-de97-4d1d-b1f0-d396daa81620",
   "metadata": {},
   "source": [
    "### 3. 模型部署准备（entrypoint脚本，容器镜像，服务配置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a198f31e-f164-4a82-a92d-a787037231fa",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image going to be used is ---- > 763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.22.1-deepspeed0.9.2-cu118\n"
     ]
    }
   ],
   "source": [
    "inference_image_uri = image_uris.retrieve(\n",
    "    framework=\"djl-deepspeed\",\n",
    "    region=sess.boto_session.region_name,\n",
    "    version=\"0.22.1\"\n",
    ")\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bc3ef47c-eaf8-46f9-ac07-464a680ca84c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!mkdir -p LLM_visualglm_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "30de600a-4877-4a32-a866-7b112d2ac99b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_visualglm_deploy_code/model.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_visualglm_deploy_code/model.py\n",
    "from djl_python import Input, Output\n",
    "import torch\n",
    "import logging\n",
    "import math\n",
    "import os\n",
    "import requests\n",
    "\n",
    "\n",
    "from transformers import AutoModel, AutoTokenizer\n",
    "model = None\n",
    "tokenizer = None\n",
    "\n",
    "DEVICE = \"cuda\"\n",
    "DEVICE_ID = \"0\"\n",
    "CUDA_DEVICE = f\"{DEVICE}:{DEVICE_ID}\" if DEVICE_ID else DEVICE\n",
    "def torch_gc():\n",
    "    if torch.cuda.is_available():\n",
    "        with torch.cuda.device(CUDA_DEVICE):\n",
    "            torch.cuda.empty_cache()\n",
    "            torch.cuda.ipc_collect()\n",
    "            \n",
    "            \n",
    "def parse_text(text):\n",
    "    \"\"\"copy from https://github.com/GaiZhenbiao/ChuanhuChatGPT/\"\"\"\n",
    "    lines = text.split(\"\\n\")\n",
    "    lines = [line for line in lines if line != \"\"]\n",
    "    count = 0\n",
    "    for i, line in enumerate(lines):\n",
    "        if \"```\" in line:\n",
    "            count += 1\n",
    "            items = line.split('`')\n",
    "            if count % 2 == 1:\n",
    "                lines[i] = f'<pre><code class=\"language-{items[-1]}\">'\n",
    "            else:\n",
    "                lines[i] = f'<br></code></pre>'\n",
    "        else:\n",
    "            if i > 0:\n",
    "                if count % 2 == 1:\n",
    "                    line = line.replace(\"`\", \"\\`\")\n",
    "                    line = line.replace(\"<\", \"&lt;\")\n",
    "                    line = line.replace(\">\", \"&gt;\")\n",
    "                    line = line.replace(\" \", \"&nbsp;\")\n",
    "                    line = line.replace(\"*\", \"&ast;\")\n",
    "                    line = line.replace(\"_\", \"&lowbar;\")\n",
    "                    line = line.replace(\"-\", \"&#45;\")\n",
    "                    line = line.replace(\".\", \"&#46;\")\n",
    "                    line = line.replace(\"!\", \"&#33;\")\n",
    "                    line = line.replace(\"(\", \"&#40;\")\n",
    "                    line = line.replace(\")\", \"&#41;\")\n",
    "                    line = line.replace(\"$\", \"&#36;\")\n",
    "                lines[i] = \"<br>\"+line\n",
    "    text = \"\".join(lines)\n",
    "    return text\n",
    "\n",
    "def load_model(properties):\n",
    "    global tokenizer,model\n",
    "    tensor_parallel = properties[\"tensor_parallel_degree\"]\n",
    "    model_location = properties['model_dir']\n",
    "    if \"model_id\" in properties:\n",
    "        model_location = properties['model_id']\n",
    "    logging.info(f\"Loading model in {model_location}\")\n",
    "    \n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_location, trust_remote_code=True)\n",
    "   \n",
    "    model = AutoModel.from_pretrained(model_location, trust_remote_code=True).half().cuda()\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    return model, tokenizer\n",
    "            \n",
    "\n",
    "\n",
    "def handle(inputs: Input):\n",
    "    global model, tokenizer\n",
    "    if not model:\n",
    "        model, tokenizer = load_model(inputs.get_properties())\n",
    "    \n",
    "    if inputs.is_empty():\n",
    "        return None\n",
    "    data = inputs.get_as_json()\n",
    "    \n",
    "    input_sentences = data[\"inputs\"]\n",
    "    image_path=data.get(\"image\",'')\n",
    "    # if image_path is None:\n",
    "    #     return  Output().add_as_json({ \"outputs\":\"图片不能为空。请重新上传图片并重试。\"})\n",
    "    ##测试链接是否能下载\n",
    "    # try:\n",
    "    #     requests.get(image_path, timeout=10)\n",
    "    # except:\n",
    "    #     return  Output().add_as_json({ \"outputs\":f'cannot download file from image_path:{image_path}'})\n",
    "\n",
    "    params = data[\"parameters\"]\n",
    "    history = data[\"history\"]\n",
    "    print(f'input prompt:{input_sentences}')\n",
    "    \n",
    "    response, history = model.chat(tokenizer,image_path, input_sentences, history=history)\n",
    "    result = {\"outputs\": parse_text(response)}\n",
    "    return Output().add_as_json(result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4568a510-2f5f-45a7-80a7-9ecd17c12153",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "option.s3url ==> s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/\n"
     ]
    }
   ],
   "source": [
    "print(f\"option.s3url ==> s3://{bucket}/{s3_model_prefix}/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a92c7a0c-6ba9-4cbe-ae4e-dfc5ed5a4368",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_visualglm_deploy_code/serving.properties\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_visualglm_deploy_code/serving.properties\n",
    "engine=Python\n",
    "option.tensor_parallel_degree=1\n",
    "option.s3url=s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_stream_model/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "cead651f-fd3c-4cf2-a87c-7b251bcc4f35",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_visualglm_deploy_code/requirements.txt\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_visualglm_deploy_code/requirements.txt\n",
    "transformers==4.30.2\n",
    "SwissArmyTransformer==0.3.6\n",
    "accelerate==0.20.3\n",
    "cpm_kernels==1.0.11\n",
    "torchvision==0.15.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "a9f1dce7-8f41-4b2e-a6b6-179c3cdc21be",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LLM_visualglm_deploy_code/\n",
      "LLM_visualglm_deploy_code/model.py\n",
      "LLM_visualglm_deploy_code/requirements.txt\n",
      "LLM_visualglm_deploy_code/serving.properties\n"
     ]
    }
   ],
   "source": [
    "!rm model.tar.gz\n",
    "!cd LLM_visualglm_deploy_code && rm -rf \".ipynb_checkpoints\"\n",
    "!tar czvf model.tar.gz LLM_visualglm_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "ba089782-5b2c-4b1a-bd31-70ff5e8d68b8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "S3 Code or Model tar ball uploaded to --- > s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_visualglm_deploy_code/model.tar.gz\n"
     ]
    }
   ],
   "source": [
    "s3_code_artifact = sess.upload_data(\"model.tar.gz\", bucket, s3_code_prefix)\n",
    "print(f\"S3 Code or Model tar ball uploaded to --- > {s3_code_artifact}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03f8d87b-1986-4d35-acc7-445b2f46cb10",
   "metadata": {},
   "source": [
    "### 4. 创建模型 & 创建endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "1b736cdb-1430-4eb1-a04c-8812f2e908dc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "visualglm-2023-08-11-09-08-07-294\n",
      "Image going to be used is ---- > 763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.22.1-deepspeed0.9.2-cu118\n",
      "Created Model: arn:aws:sagemaker:us-east-2:946277762357:model/visualglm-2023-08-11-09-08-07-294\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.utils import name_from_base\n",
    "import boto3\n",
    "\n",
    "model_name = name_from_base(f\"visualglm\") # Append a timestamp to the provided string\n",
    "print(model_name)\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")\n",
    "\n",
    "create_model_response = sm_client.create_model(\n",
    "    ModelName=model_name,\n",
    "    ExecutionRoleArn=role,\n",
    "    PrimaryContainer={\n",
    "        \"Image\": inference_image_uri,\n",
    "        \"ModelDataUrl\": s3_code_artifact\n",
    "    },\n",
    "    \n",
    ")\n",
    "model_arn = create_model_response[\"ModelArn\"]\n",
    "\n",
    "print(f\"Created Model: {model_arn}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "85794eff-c9b7-44fd-85fa-a3ec40416e8e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'EndpointConfigArn': 'arn:aws:sagemaker:us-east-2:946277762357:endpoint-config/visualglm-2023-08-11-09-08-07-294-config',\n",
       " 'ResponseMetadata': {'RequestId': '145d789b-db76-4fb4-9c61-4f41082b38e8',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': '145d789b-db76-4fb4-9c61-4f41082b38e8',\n",
       "   'content-type': 'application/x-amz-json-1.1',\n",
       "   'content-length': '121',\n",
       "   'date': 'Fri, 11 Aug 2023 09:08:08 GMT'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "endpoint_config_name = f\"{model_name}-config\"\n",
    "endpoint_name = f\"{model_name}-endpoint\"\n",
    "\n",
    "#Note: ml.g4dn.2xlarge 也可以选择\n",
    "endpoint_config_response = sm_client.create_endpoint_config(\n",
    "    EndpointConfigName=endpoint_config_name,\n",
    "    ProductionVariants=[\n",
    "        {\n",
    "            \"VariantName\": \"variant1\",\n",
    "            \"ModelName\": model_name,\n",
    "            \"InstanceType\": \"ml.g5.2xlarge\",\n",
    "            \"InitialInstanceCount\": 1,\n",
    "            # \"VolumeSizeInGB\" : 400,\n",
    "            # \"ModelDataDownloadTimeoutInSeconds\": 2400,\n",
    "            \"ContainerStartupHealthCheckTimeoutInSeconds\": 10*60,\n",
    "        },\n",
    "    ],\n",
    ")\n",
    "endpoint_config_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "3d3bd72c-eddc-40d2-8805-372551efb1b6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created Endpoint: arn:aws:sagemaker:us-east-2:946277762357:endpoint/visualglm-2023-08-11-09-08-07-294-endpoint\n"
     ]
    }
   ],
   "source": [
    "create_endpoint_response = sm_client.create_endpoint(\n",
    "    EndpointName=f\"{endpoint_name}\", EndpointConfigName=endpoint_config_name\n",
    ")\n",
    "print(f\"Created Endpoint: {create_endpoint_response['EndpointArn']}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "ae08ecbc-4141-401f-b60e-d76bf9c6900e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: InService\n",
      "Arn: arn:aws:sagemaker:us-east-2:946277762357:endpoint/visualglm-2023-08-11-09-08-07-294-endpoint\n",
      "Status: InService\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "status = resp[\"EndpointStatus\"]\n",
    "print(\"Status: \" + status)\n",
    "\n",
    "while status == \"Creating\":\n",
    "    time.sleep(60)\n",
    "    resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "    status = resp[\"EndpointStatus\"]\n",
    "    print(\"Status: \" + status)\n",
    "\n",
    "print(\"Arn: \" + resp[\"EndpointArn\"])\n",
    "print(\"Status: \" + status)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d4fc30d-ddc3-4b27-bf0b-a79a694a94e3",
   "metadata": {},
   "source": [
    "### 模型测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a91e80ea-cd26-4c13-bb19-29cba6a2021a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import boto3\n",
    "\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "d85e2d6e-eed6-44af-b3b6-c6defa1f0570",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import boto3\n",
    "def generate_s3_image_url(bucket_name, key, expiration=3600):\n",
    "    s3_client = boto3.client('s3')\n",
    "    url = s3_client.generate_presigned_url(\n",
    "        'get_object',\n",
    "         Params={'Bucket': bucket_name, 'Key': key},\n",
    "         ExpiresIn=expiration\n",
    "    )\n",
    "    return url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "9cdee7d2-96a1-4d6b-81ff-ee3d47aa55de",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://946277762357-23-07-24-16-11-59-bucket.s3.amazonaws.com/images/cat_dog.jpeg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ASIA5YUUMYU26C623W6G%2F20230810%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20230810T150900Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Security-Token=IQoJb3JpZ2luX2VjEBcaCXVzLWVhc3QtMiJHMEUCIQCjzokFgjUwlmSBNq8PuQcyx6rq8woo%2F0H%2BwFhDqdVgeQIgenSoN5TYTPVfgZmHu%2BTqNYcePUs69R7WcN%2BPNpBlUDAqigMIwP%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FARABGgw5NDYyNzc3NjIzNTciDF572NvpjblTbw%2BYvireAumLPMdV2urut9W9R687v%2FiN9bWlBnF3JLqov7qnBiFnnKan4hPt6u5IOLouM5IA%2B%2FsD1XrAxUTjIFjd1mVlfTpQkptBvlepG9btMwyyRpuY2anZAtVzvRoi2PTbdL7JQ%2BNOTh%2F5Mno2cMTO5eWHPCgHkzuuJ1%2Fed%2B2cvYPJhbVBcJH9RdEcw0mykLLYLKEQMVAkzInQ1gBGFKW%2BlBrdhapRFI8mKI0dXWgWVpcJs7%2FiRdJNCrW%2BCc33i8O%2FmMUMM4ZHWu2NwRMI45UIqykDbyfEM0l2XqgUx2F8eC%2B%2BYf%2F8ILD3X33fEhHp29AOPZa8ORWIIRMRw1AEb5sBjeTDm4kPrMKWICbuMRaHegAGZixgcBdVnsrwSoADDW2%2BCohErQuwWRvEG%2F7r1dbFIEKfw3RDgtwtAZgu7ZIfDf%2BZQLYVChD5tyP0jkzSr9Eia5VourQj%2BikwbmYkY%2F8Gm1pBMOvv06YGOpsBhtP9wIsrwTnLf4jvz%2Fag5o8zswUJBk9H7i%2B%2F29A7j6aKGVfe%2F%2Bn2c7BbJbSMM%2BGyukVbEkRYJoun8uFk%2BmDEjFYdUjWOPBE9W6q8%2BF2loYGWylyqw02%2BE9zAvWPWXulwq1pilAoEKJInNcTkowG2tz1Ml%2B8uTf5Wa3lsPpG%2BZ%2FEzftuxUB6mrGMc52DdZ74t5hkhUXgNeZgTWvQ%3D&X-Amz-Signature=4c3cfc3d1d13f07f54163c0d1162904e4ddcfffc63629257b1535d7484d849f7'"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imgurl='946277762357-23-07-24-16-11-59-bucket/images/cat_dog.jpeg'\n",
    "bucket,imgobj = imgurl.split('/',1)\n",
    "image_path = generate_s3_image_url(bucket,imgobj)\n",
    "image_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "35de17d6-d74c-44e6-bfb0-ebd7cb659c00",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "history = [('请描述这张图片','这张照片展示了一只棕色金毛犬和一只灰色猫坐在地板上。它们看起来很放松，似乎享受彼此的陪伴。图片中有一只狗的耳朵在猫旁边，这暗示着它们之间可能存在某种联系或互动。然而，这只猫看起来有些警惕，可能正在观察狗的行为或者试图避免任何潜在的威胁。')]\n",
    "prompt = \"将金毛犬取名小金，灰色猫取名小灰，并写成一首诗\"\n",
    "image_path = \"http://images.cocodataset.org/val2017/000000039769.jpg\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "eb1867c5-8f02-40c5-a72a-9acc13532b98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\\n  \"outputs\":\"小金和小灰，两只可爱的猫咪，  睡在同一张床上。它们躺在沙发上，  像两个枕头一样亲密。 小金和小灰，彼此拥抱着，  一起呼吸着温暖的气息。它们的毛皮柔软，  在柔和的光线下闪闪发光。 它们似乎享受彼此的存在，  就像两只温暖的伴侣。 小金和小灰，如此和谐，  让我们感到幸福和温馨。它们在我们周围休息，  给我们带来宁静与平静。 当它们醒来时，它们会相互玩耍，  用爪子抓住对方，发出欢快的笑声。 小金和小灰，如此可爱，  为我们带来了快乐和爱。\"\\n}'"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parameters = {\n",
    "  \"max_length\": 2000,\n",
    "  \"temperature\": 0.1,\n",
    "  \"top_p\":1\n",
    "}\n",
    "\n",
    "response_model = smr_client.invoke_endpoint(\n",
    "            EndpointName=endpoint_name,\n",
    "            Body=json.dumps(\n",
    "            {\n",
    "                \"inputs\": prompt,\n",
    "                \"image\":image_path,\n",
    "                \"parameters\": parameters,\n",
    "                \"history\" : []\n",
    "            }\n",
    "            ),\n",
    "            ContentType=\"application/json\",\n",
    "        )\n",
    "\n",
    "response_model['Body'].read().decode('utf8')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90fa3ac8-c44b-472b-99c3-29d677b53d7f",
   "metadata": {},
   "source": [
    "#### 清除模型Endpoint和config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "9d253f55-082c-40ab-b544-0a91f758fe39",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws sagemaker delete-endpoint --endpoint-name  {endpoint_name}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "43e77cb5-447b-4f79-b13f-35a806f150f9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws sagemaker delete-endpoint-config --endpoint-config-name {endpoint_config_name}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "8e1b382f-a155-40dd-8f5e-0fc3300c9050",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws sagemaker delete-model --model-name  {model_name}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e939dbd4-5a3d-4b91-a777-089315ee6848",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   }
  ],
  "instance_type": "ml.g4dn.2xlarge",
  "kernelspec": {
   "display_name": "Python 3 (PyTorch 2.0.0 Python 3.10 GPU Optimized)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-2:429704687514:image/pytorch-2.0.0-gpu-py310"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
